package com.siwei.infoextract.web;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TimerTask;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import ahxu.crawler.CrawlServiceProvider;
import ahxu.crawler.CrawlerResultDaoException;
import ahxu.crawler.CrawlerSaveResult;
import ahxu.crawler.ICrawlerResutlDao;
import ahxu.search.db.FinalResult;
import ahxu.search.db.FinalResultDaoException;
import ahxu.search.db.IFinalResultDao;
import ahxu.search.db.IUnFinalResultDao;
import ahxu.search.db.SearchServiceProvider;

import com.siwei.infoextract.rule.RuleEngine;
import com.siwei.infoextract.util.Utility;
import com.siwei.infoextract.wrapper.InfoBlocksExtractor;
import com.siwei.infoextract.wrapper.featureextract.InnerTextFeatureExtractor;
import com.siwei.infoextract.wrapper.parser.DomNode;
import com.siwei.infoextract.wrapper.parser.DomNodeIteratorCallback;
import com.siwei.infoextract.wrapper.parser.DomTreeTraversal;

/**
 * We can call this with a timer
 * @author Administrator
 *
 */
public class InfoProcessor extends TimerTask {
	Log logger = LogFactory.getLog(InfoProcessor.class);

	private static final int HANDLE_RECORDS_COUNT_EVERY_TIME = 100;
	private static final int UNSTRUCT_INFO_DETECT_THRESHOLD = 300;

	public static boolean isRunning = false;

	private static Pattern stylePattern = Pattern.compile("<style.*?>.*?</style>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
	private static Pattern scriptPattern = Pattern.compile("<script.*?>.*?</script>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
	private static Pattern formPattern = Pattern.compile("<form.*?>.*?</form>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
	private static Pattern linkPattern = Pattern.compile("<a.*?>.*?</a>", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
	private static Pattern tagPattern = Pattern.compile("<.*?>", Pattern.DOTALL);

	private static String[] eventObjects = {
		"道路",
		"地铁",
		"机场",
		"宾馆",
		"旅馆",
		"酒店",
		"连锁店",
		"商场"
	};
	private static String[] eventActions = {
		"开通",
		"竣工",
		"维修",
		"施工",
		"动土",
		"开业",
		"停业",
		"整顿"
	};

	public void run() {
		logger.info("Info process starting to run");
		if (isRunning) {
			logger.info("There is already another process is running, skip this run");
			return;
		}

		// lock all the other call
		//
		isRunning = true;

		try {
			// Get craolwer result
			ICrawlerResutlDao crawlerResutlDao = CrawlServiceProvider.getInstance().getResultDao();
			IFinalResultDao finalResultDao = SearchServiceProvider.getInstance().getFinalResultDao();
			IUnFinalResultDao unFinalResultDao = SearchServiceProvider.getInstance().getUnFinalResultDao();

			List<CrawlerSaveResult> unhandledPages = null;
			try {
				unhandledPages = crawlerResutlDao.findUndoneResultsByTop(HANDLE_RECORDS_COUNT_EVERY_TIME);
			} catch (CrawlerResultDaoException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

			if (unhandledPages == null) {
				return;
			}

			for (Iterator<CrawlerSaveResult> saveResultIter = unhandledPages.iterator(); saveResultIter.hasNext();) {
				CrawlerSaveResult crawlerSaveResult = saveResultIter.next();
				try {
					extractStructedInfo(finalResultDao, crawlerSaveResult);
					extractUnstrunctedInfo(unFinalResultDao, crawlerSaveResult);
				} finally {
					try {
						crawlerResutlDao.updateResultDone(crawlerSaveResult.getId(), true);
					} catch (CrawlerResultDaoException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
				}
			}
			logger.info("Info process run completed");
		} finally {
			isRunning = false;
		}
	}

	private void extractUnstrunctedInfo(IUnFinalResultDao unFinalResultDao, CrawlerSaveResult crawlerSaveResult) {
		try {
			String filePath = crawlerSaveResult.getSourceFile();
			String content = Utility.getContentFromFile(filePath);
	
			String cleanedContent = cleanHtmlContent(content);
			
			if (cleanedContent.length() > UNSTRUCT_INFO_DETECT_THRESHOLD) {
				// Save to unstructed info table
				for(int j = 0; j < eventObjects.length; j++) {
					for (int k = 0; k < eventActions.length; k++) {
						if (cleanedContent.indexOf(eventObjects[j]) > -1 && cleanedContent.indexOf(eventActions[k]) > -1) {
							// Here we found an unstructed page, we need to save it to the database
							try {
								unFinalResultDao.save(crawlerSaveResult.getId(), eventObjects[j], eventActions[k]);
							} catch (FinalResultDaoException frde) {
								// We skip the exception here
							}
						}
					}
				}
			}
		} catch (IOException ioe) {
			// Skip the exception here
		}
	}

	private String cleanHtmlContent(String content) {
		// Remove all styles
		//
		StringBuffer stringBuffer = new StringBuffer();
		Matcher matcher = stylePattern.matcher(content);
		while(matcher.find()) {
			matcher.appendReplacement(stringBuffer, "");
		}
		matcher.appendTail(stringBuffer);
		
		// Remove all scripts
		//
		String cleanedString = stringBuffer.toString();
		stringBuffer.delete(0, stringBuffer.length());
		matcher = scriptPattern.matcher(cleanedString);
		while(matcher.find()) {
			matcher.appendReplacement(stringBuffer, "");
		}
		matcher.appendTail(stringBuffer);

		// Remove all forms
		//
		cleanedString = stringBuffer.toString();
		stringBuffer.delete(0, stringBuffer.length());
		matcher = formPattern.matcher(cleanedString);
		while(matcher.find()) {
			matcher.appendReplacement(stringBuffer, "");
		}
		matcher.appendTail(stringBuffer);
		
		// Remove all links
		//
		cleanedString = stringBuffer.toString();
		stringBuffer.delete(0, stringBuffer.length());
		matcher = linkPattern.matcher(cleanedString);
		while(matcher.find()) {
			matcher.appendReplacement(stringBuffer, "");
		}
		matcher.appendTail(stringBuffer);

		// Remove all tags
		//
		cleanedString = stringBuffer.toString();
		stringBuffer.delete(0, stringBuffer.length());
		matcher = tagPattern.matcher(cleanedString);
		while(matcher.find()) {
			matcher.appendReplacement(stringBuffer, "");
		}
		matcher.appendTail(stringBuffer);

		return stringBuffer.toString();
	}


	private void extractStructedInfo(IFinalResultDao finalResultDao,
			CrawlerSaveResult crawlerSaveResult) {
		String filePath = crawlerSaveResult.getSourceFile();

		try {
			DomNode root = Utility.parseDomNodeFromFile(filePath);

			// Extract info blocks
			InfoBlocksExtractor extractor = new InfoBlocksExtractor();
			List<DomNode> results = extractor.extract(root);

			// Get title/address/phone text
			final List<FinalResult> structInfoList = new ArrayList<FinalResult>();
			
			final InnerTextFeatureExtractor innerTextExtractor = new InnerTextFeatureExtractor();

			for (Iterator<DomNode> iter = results.iterator(); iter
					.hasNext();) {
				DomTreeTraversal traversal = new DomTreeTraversal(iter.next());

				final FinalResult infoBlock = new FinalResult();
				infoBlock.setCrawlId(crawlerSaveResult.getId());
				infoBlock.setUrl(crawlerSaveResult.getUrl()); // a redudant field

				traversal.addCallback(new DomNodeIteratorCallback() {
					public void call(DomNode domNode) {
						Map<String, String> map = (Map<String, String>) domNode.getFlag();

						String innerText = (String) innerTextExtractor
								.extract(domNode);
						if (map.get(RuleEngine.TITLE_RULE_ENGINE_NAME) != null) {
							boolean isTitle = Boolean.parseBoolean(map
									.get(RuleEngine.TITLE_RULE_ENGINE_NAME));
							if (isTitle) {
								infoBlock.setName(innerText);
							}
						}

						if (map.get(RuleEngine.ADDRESS_RULE_ENGINE_NAME) != null) {
							boolean isAddress = Boolean.parseBoolean(map
									.get(RuleEngine.ADDRESS_RULE_ENGINE_NAME));
							if (isAddress) {
								infoBlock.setAddress(innerText);
							}
						}

						if (map.get(RuleEngine.PHONE_RULE_ENGINE_NAME) != null) {

							boolean isPhone = Boolean.parseBoolean(map
									.get(RuleEngine.PHONE_RULE_ENGINE_NAME));
							if (isPhone) {
								infoBlock.setPhone(innerText);
							}
						}
					}
				});
				traversal.traverse();

				structInfoList.add(infoBlock);
			}

			// Save to structed info table
			for (Iterator<FinalResult> infoBlockIter = structInfoList.iterator(); infoBlockIter.hasNext();) {
				try {
					finalResultDao.save(infoBlockIter.next());
				} catch (FinalResultDaoException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		} catch (IOException ioe) {
			// Skip the exception here
		}
	}

}
